# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html


# useful for handling different item types with a single interface
from itemadapter import ItemAdapter

from scrapy.exceptions import DropItem

import pymongo


class TextPipeline:
    def __init__(self):
        # 限制text最长长度为50
        self.limit = 50

    def process_item(self, item, spider):
        if item.get('text', None):
            if len(item['text']) > self.limit:
                # 清理字符串首位空格
                text = item['text'].strip()
                text = text[0: self.limit] + '...'
                item['text'] = text
            return item
        else:
            return DropItem("Text字段缺失")


class MongoPipeline:
    def __init__(self, mongo_uri, mongo_db):
        self.mongo_uri = mongo_uri
        self.mongo_db = mongo_db

    @classmethod
    def from_crawler(cls, crawler):
        # 实例化 MongoPipeline 类
        db = cls(mongo_uri=crawler.settings.get('MONGO_URI'),
                 mongo_db=crawler.settings.get('MONGO_DB'))
        return db

    def open_spider(self, spider):
        self.client = pymongo.MongoClient(self.mongo_uri)
        self.db = self.client[self.mongo_db]

    def process_item(self, item, spider):
        # item名：
        name = item.__class__.__name__
        # 为每个item创建一个mongo db，将不同item中对应的数据插入到对应的db中
        self.db[name].insert(dict(item))
        return item

    def close_spider(self, spider):
        # 关闭mongo链接
        self.client.close()

